/*
########################################################################
#  Netflix Prize Tools
#  Copyright (C) 2007-8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
*/
/*
 * Read the binary files generated by netflix2moviebin.py and convert
 * them to user oriented files.
 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "basic.h"
#include "netflix.h"

static char *useridx_path="data/user_index.bin";
static char *userent_path="data/user_entry.bin";

int movieidx[NMOVIES][4];
unsigned int movieent[NENTRIES];
unsigned short moviedat[NENTRIES];

int useridx[NUSERS][4];
int usercount[NUSERS][4]; // we dont use [0]
unsigned int userent[NENTRIES];

int cmp(const void *p1, const void *p2)
{
	unsigned int e1=*(unsigned int *)p1;
	unsigned int e2=*(unsigned int *)p2;
	if(e1<e2) return -1;
	if(e1>e2) return 1;
	return 0;
}

main() {
	load_bin(movieidx_path,movieidx,sizeof(movieidx));
	load_bin(movieent_path,movieent,sizeof(movieent));
	load_bin(moviedate_path,moviedat,sizeof(moviedat));
	
	lg("Computing user index\n");
    ZERO(useridx);
	int m,j,k;
    for(m=0; m<NMOVIES; m++) {
		int base=movieidx[m][0];
		for(k=1;k<4;k++)
        for(j=0;j<movieidx[m][k];j++) {
            unsigned int dd=movieent[base++];
            int u=dd&MOVIE_USERMASK;
			useridx[u][k]++;
        }
    }
	useridx[0][0]=0;
	int u;
	for(u=0;u<NUSERS-1;u++)
		for(k=0;k<4;k++)
			useridx[u+1][0]+=useridx[u][k];
	dump_bin(useridx_path,useridx,sizeof(useridx));
	
	{
		int count[4];
		ZERO(count);
		for(u=0;u<NUSERS-1;u++)
			for(k=1;k<4;k++)
				count[k]+=useridx[u][k];
		lg("Train=%d Probe=%d Qualify=%d\n",count[1],count[2],count[3]);
		u=52;
		lg("%d %d %d %d\n",useridx[u][0],useridx[u][1],useridx[u][2],useridx[u][3]);
	}
	
	/* convert counts to idx */
	for(u=0;u<NUSERS;u++)
		for(k=1;k<4;k++)
			useridx[u][k]+=useridx[u][k-1];
			
	lg("Test idx\n");	
	for(u=0;u<NUSERS;u++)
		if(useridx[u][4]!=useridx[u+1][0])
			error("Error");
			
	ZERO(usercount);
    for(m=0; m<NMOVIES; m++) {
		int base=movieidx[m][0];
		for(k=1;k<4;k++)
        for(j=0;j<movieidx[m][k];j++) {
			unsigned short dat = moviedat[base];
            unsigned int dd=movieent[base++];
            int u=dd&MOVIE_USERMASK;
			int r=(dd>>MOVIE_LUSERMASK)&7;
			dd=m | (r<<USER_LMOVIEMASK) | (days(dat>>9,(dat>>5)&15,dat&31)<<(USER_LMOVIEMASK+3));
			int idx=usercount[u][k]++;
			userent[useridx[u][k-1]+idx]=dd;
        }
    }
	lg("Test user count\n");
	for(u=0;u<NUSERS;u++)
		for(k=0;k<3;k++) {
			if(usercount[u][k+1]!=useridx[u][k+1]-useridx[u][k])
				error("Error");
	}
	lg("Sorting result\n");
	int errcount=0;
	for(u=0;u<NUSERS;u++)
		for(k=0;k<3;k++) {
			int n=usercount[u][k+1];
			int base=useridx[u][k];
			if(n>1)
				qsort(&userent[base],n,sizeof(unsigned int),cmp);
			for(j=0;j<n-1;j++) {
				unsigned int x1=userent[base+j];
				unsigned int x2=userent[base+j+1];
				
				if(x1>x2) {
					lg("Error %d+%d out of %d %x %x\n",base,j,n,x1,x2);
					if(errcount++>50) exit(1);
				}
			}
		}
	lg("Test sort\n");
	for(u=0;u<NUSERS;u++)
		for(k=0;k<3;k++)
			for(j=0;j<usercount[u][k+1]-1;j++)
				if(userent[useridx[u][k]+j]>userent[useridx[u][k]+j+1])
					error("Error");
	{
		u=52;
		int k=3,i;
		int base=useridx[u][k-1];
		int d=usercount[u][k];
		for(i=0; i<d;i++) {
			int day=userent[base+i]>>(USER_LMOVIEMASK+3);
			lg("%d %d\n",base+i,day);
		}
	}			
	dump_bin(userent_path,userent,sizeof(userent));	
}
